Stable Diffusion
Contents
Stable Diffusion#
Note
Install ekorpkit package first.
Set logging level to Warning, if you don’t want to see verbose logging.
If you run this notebook in Colab, set Hardware accelerator to GPU.
%pip install -U --pre ekorpkit[art]
exit()
Prepare Environment#
%config InlineBackend.figure_format='retina'
%load_ext autotime
from ekorpkit import eKonf
eKonf.setLogger("INFO")
print("version:", eKonf.__version__)
is_colab = eKonf.is_colab()
print("is colab?", is_colab)
if is_colab:
eKonf.mount_google_drive()
workspace_dir = "/content/drive/MyDrive/workspace"
project_name = "ekorpkit-book"
project_dir = eKonf.set_workspace(workspace=workspace_dir, project=project_name)
print("project_dir:", project_dir)
eKonf.os.envs.dict()
INFO:ekorpkit.utils.notebook:Google Colab not detected.
INFO:ekorpkit.base:Set environment variable EKORPKIT_WORKSPACE_ROOT=/content/drive/MyDrive/workspace
INFO:ekorpkit.base:Set environment variable EKORPKIT_PROJECT_DIR=/content/drive/MyDrive/workspace/projects/ekorpkit-book
version: 0.1.40.post0.dev45
is colab? False
project_dir: /content/drive/MyDrive/workspace/projects/ekorpkit-book
{'EKORPKIT_CONFIG_DIR': '/workspace/projects/ekorpkit-book/config',
'EKORPKIT_WORKSPACE_ROOT': '/content/drive/MyDrive/workspace',
'EKORPKIT_PROJECT': 'ekorpkit-book',
'EKORPKIT_PROJECT_DIR': '/content/drive/MyDrive/workspace/projects/ekorpkit-book',
'EKORPKIT_DATA_DIR': None,
'EKORPKIT_LOG_LEVEL': 'INFO',
'NUM_WORKERS': 230,
'KMP_DUPLICATE_LIB_OK': 'TRUE',
'CUDA_DEVICE_ORDER': None,
'CUDA_VISIBLE_DEVICES': None,
'WANDB_PROJECT': None}
time: 1.08 s (started: 2022-12-05 05:03:13 +00:00)
Create a stable diffusion instance#
To download a certain dataset or model checkpoint, you may need to provide a HuggingFace API token. You can get one from here.
# Set HuggingFace API token
eKonf.os.secrets.HUGGING_FACE_HUB_TOKEN = "YOUR_TOKEN"
# Set HuggingFace API token
# eKonf.secrets().HUGGING_FACE_HUB_TOKEN = "YOUR_TOKEN"
eKonf.os.secrets.dict()
{'WANDB_API_KEY': SecretStr('**********'),
'HUGGING_FACE_HUB_TOKEN': None,
'ECOS_API_KEY': SecretStr('**********'),
'FRED_API_KEY': SecretStr('**********'),
'NASDAQ_API_KEY': SecretStr('**********'),
'HF_USER_ACCESS_TOKEN': SecretStr('**********')}
time: 2.34 ms (started: 2022-12-05 05:03:14 +00:00)
# Set CUDA DEVICES for the model
eKonf.os.envs.CUDA_VISIBLE_DEVICES = "1,2"
INFO:ekorpkit.base:Set environment variable CUDA_VISIBLE_DEVICES=1,2
time: 1.25 ms (started: 2022-12-05 05:03:14 +00:00)
from ekorpkit.tasks.multi import StableDiffusion
sd = StableDiffusion()
INFO:absl:Unable to initialize backend 'tpu_driver': NOT_FOUND: Unable to find driver in registry given worker:
INFO:absl:Unable to initialize backend 'rocm': NOT_FOUND: Could not find registered platform with name: "rocm". Available platform names are: Interpreter CUDA Host
INFO:absl:Unable to initialize backend 'tpu': module 'jaxlib.xla_extension' has no attribute 'get_tpu_client'
INFO:root:compose config with overrides: ['+task/multi/text2image=stable-diffusion']
INFO:ekorpkit.config:Using existing path: /content/drive/MyDrive/workspace/projects/ekorpkit-book/aiart
INFO:ekorpkit.base:Set environment variable HUGGING_FACE_HUB_TOKEN=**********
INFO:ekorpkit.config:No modules to load
time: 5.99 s (started: 2022-12-05 05:03:28 +00:00)
Generate images#
text_prompts = "people looking out the lonely city street from windows of buildings in a Edward Hopper style. detailed, romantic, enchanting, trending on artstation."
batch_name = "hopper-street"
sd.config.collage.ncols = 2
sd.config.batch.verbose = True
results = sd.generate(
text_prompts,
batch_name=batch_name,
num_samples=4,
guidance_scale=9,
width=800,
height=400,
num_inference_steps=50,
)
Prompt: people looking out the lonely city street from windows of buildings in a Edward Hopper style. detailed, romantic, enchanting, trending on artstation.
time: 48.1 s (started: 2022-12-05 05:03:40 +00:00)
base_url = "https://github.com/entelecheia/ekorpkit-book/raw/main/assets/figs"
init_image_path = f"{base_url}/chu-horse.png"
batch_name = "horse-init"
text_prompts = "a golden statue of an eagle with clouds"
sd.config.collage.ncols = 2
results = sd.generate(
text_prompts,
batch_name=batch_name,
init_image=init_image_path,
num_samples=4,
guidance_scale=30,
width=800,
height=400,
num_inference_steps=50,
)
time: 457 µs (started: 2022-12-02 21:36:54 +00:00)
Inpaint images#
base_url = "https://github.com/entelecheia/ekorpkit-book/raw/main/assets/figs"
init_image_path = f"{base_url}/chu-horse.png"
mask_image_path = f"{base_url}/chu-horse-masking-rough.png"
mask_image_exact_path = f"{base_url}/chu-horse-masking-exact.png"
init_image = eKonf.load_image(init_image_path)
mask_image = eKonf.load_image(mask_image_path)
mask_image_exact = eKonf.load_image(mask_image_exact_path)
sd.collage_images([mask_image, init_image, mask_image_exact])
Prompt: people looking out the lonely city street from windows of buildings in a Edward Hopper style. detailed, romantic, enchanting, trending on artstation.
time: 2.48 s (started: 2022-11-30 01:57:19 +00:00)
prompt = "a golden statue of an eagle with clouds, colorful painting by Tanya Hern, mural on the roof, digital art, artwork, beautiful, colorful, visual art on landscape, surrealism, watercolor, vivid by stunning"
batch_name = "inpaint-halla"
sd.config.collage.ncols = 3
results = sd.generate(
prompt,
batch_name=batch_name,
mode="inpaint",
init_image=init_image_path,
mask_image=mask_image_path,
num_samples=3,
num_images_per_prompt=1,
inpaint_strength=10,
num_inference_steps=100,
)
Prompt: a golden statue of an eagle with clouds, colorful painting by Tanya Hern, mural on the roof, digital art, artwork, beautiful, colorful, visual art on landscape, surrealism, watercolor, vivid by stunning
time: 1min 9s (started: 2022-11-30 01:57:21 +00:00)
Stitch images#
text_prompts = "people looking out the lonely city street from windows of buildings in a Edward Hopper style. detailed, romantic, enchanting, trending on artstation."
batch_name = "city-stitch"
results = sd.generate(
text_prompts,
mode="stitch",
batch_name=batch_name,
num_samples=6,
num_images_per_prompt=2,
guidance_scale=7,
inpaint_strength=10,
)
INFO:ekorpkit.models.art.stable: >> elapsed time to imagine: 0:01:47.820320
time: 1min 48s (started: 2022-11-30 01:47:30 +00:00)
To view the panorama, upload the image to renderstuff.
Collage Images#
sd.collage_images(batch_name="hopper-street", batch_num=10, ncols=2, collage_width=800)
Prompt: people looking out the lonely city street from windows of buildings in a Edward Hopper style. detailed, romantic, enchanting, trending on artstation.
time: 523 ms (started: 2022-11-30 02:23:48 +00:00)
Show Configuration#
sd.show_config(batch_name="hopper-street", batch_num=10)
INFO:ekorpkit.config:> Loading config for batch_name: hopper-street batch_num: 10
INFO:ekorpkit.config:Using existing path: /content/drive/MyDrive/workspace/projects/ekorpkit-book/aiart
INFO:ekorpkit.config:Batch name: hopper-street, Batch num: 10
INFO:ekorpkit.config:Loading config from /content/drive/MyDrive/workspace/projects/ekorpkit-book/aiart/outputs/hopper-street/configs/hopper-street(10)_config.yaml
INFO:ekorpkit.config:Merging config with the loaded config
INFO:ekorpkit.config:Merging config with args: {}
INFO:ekorpkit.config:Using existing path: /content/drive/MyDrive/workspace/projects/ekorpkit-book/aiart
INFO:ekorpkit.config:Batch name: hopper-street, Batch num: 10
{'_target_': 'ekorpkit.models.art.stable.StableDiffusion',
'auto': {},
'autoload': False,
'batch': {'batch_name': 'hopper-street',
'batch_num': 10,
'num_workers': '230',
'random_seed': True,
'resume_latest': False,
'resume_run': False,
'run_to_resume': 'latest',
'seed': None,
'verbose': True},
'collage': {'bg_color': 'black',
'clear_output': True,
'collage_width': 1200,
'crop_to_min_size': False,
'fontsize': 12,
'max_images': 12,
'ncols': 3,
'padding': 5,
'show_filename': False,
'show_prompt': True},
'device': 'cuda:0',
'imagine': {'clear_output': True,
'display_collage': True,
'display_image': True,
'guidance_scale': 9,
'height': 400,
'increase_seed_by': 1,
'init_image': None,
'inpaint_strength': 7.5,
'mask_image': None,
'mode': 'generate',
'num_images_per_prompt': 1,
'num_inference_steps': 150,
'num_samples': 4,
'save_collage': True,
'save_image': True,
'save_image_config': True,
'seed': None,
'text_prompts': 'people looking out the lonely city street from '
'windows of buildings in a Edward Hopper style. '
'detailed, romantic, enchanting, trending on '
'artstation.',
'width': 800},
'model': {'generate': {'device': 'cuda:0',
'name': 'CompVis/stable-diffusion-v1-4',
'pipeline': 'StableDiffusionPipeline',
'revision': 'fp16',
'use_auth_token': True},
'inpaint': {'device': 'cuda:0',
'name': 'runwayml/stable-diffusion-inpainting',
'pipeline': 'StableDiffusionInpaintPipeline',
'revision': 'fp16',
'use_auth_token': True}},
'module': {'library_dir': '/content/drive/MyDrive/workspace/projects/ekorpkit-book/aiart/libs',
'modules': None},
'name': 'hopper-street',
'path': {'batch_dir': '/content/drive/MyDrive/workspace/projects/ekorpkit-book/aiart/outputs/hopper-street',
'batch_name': 'hopper-street',
'cache_dir': '/content/drive/MyDrive/workspace/.cache',
'data_dir': '/content/drive/MyDrive/workspace/projects/ekorpkit-book/aiart/data',
'library_dir': '/content/drive/MyDrive/workspace/projects/ekorpkit-book/aiart/libs',
'model_dir': '/content/drive/MyDrive/workspace/projects/ekorpkit-book/aiart/models',
'output_dir': '/content/drive/MyDrive/workspace/projects/ekorpkit-book/aiart/outputs',
'root': '/content/drive/MyDrive/workspace/projects/ekorpkit-book/aiart',
'task_name': 'aiart',
'tmp_dir': '/content/drive/MyDrive/workspace/.tmp',
'verbose': False},
'project': {'description': None,
'path': {'archive': '/content/drive/MyDrive/workspace/data/archive',
'cache': '/content/drive/MyDrive/workspace/.cache',
'corpus': '/content/drive/MyDrive/workspace/data/datasets/corpus',
'data': '/content/drive/MyDrive/workspace/data',
'dataset': '/content/drive/MyDrive/workspace/data/datasets',
'ekorpkit': '/workspace/projects/ekorpkit/ekorpkit',
'home': '/root',
'library': '/content/drive/MyDrive/workspace/data/libs',
'log': '/content/drive/MyDrive/workspace/projects/ekorpkit-book/logs',
'model': '/content/drive/MyDrive/workspace/data/models',
'output': '/content/drive/MyDrive/workspace/projects/ekorpkit-book/outputs',
'project': '/content/drive/MyDrive/workspace/projects/ekorpkit-book',
'resource': '/workspace/projects/ekorpkit/ekorpkit/resources',
'runtime': '/workspace/projects/ekorpkit-book/ekorpkit-book/docs/lectures/aiart',
'tmp': '/content/drive/MyDrive/workspace/.tmp',
'workspace': '/content/drive/MyDrive/workspace'},
'project_dir': '/content/drive/MyDrive/workspace/projects/ekorpkit-book',
'project_name': 'ekorpkit-book',
'task_name': 'aiart',
'workspace_dir': '/content/drive/MyDrive/workspace'},
'secret': {'hf_user_access_token': None, 'wandb_api_key': None},
'verbose': False}
time: 367 ms (started: 2022-11-30 02:23:59 +00:00)